{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# 4. 正則化\n", "\n", "過学習が起こるのは\n", "\n", "- パラメータを大量に持ち、表現力の高いモデルである\n", "- 訓練データが少ない" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sys, os\n", "sys.path.append(os.path.abspath(os.path.join('..', 'sample')))" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "epoch:0, train acc:0.09333333333333334, test acc:0.1056\n", "epoch:1, train acc:0.10666666666666667, test acc:0.1176\n", "epoch:2, train acc:0.16, test acc:0.1435\n", "epoch:3, train acc:0.19, test acc:0.1634\n", "epoch:4, train acc:0.22666666666666666, test acc:0.1913\n", "epoch:5, train acc:0.23333333333333334, test acc:0.2093\n", "epoch:6, train acc:0.24, test acc:0.2246\n", "epoch:7, train acc:0.25333333333333335, test acc:0.2352\n", "epoch:8, train acc:0.27, test acc:0.2472\n", "epoch:9, train acc:0.28, test acc:0.2553\n", "epoch:10, train acc:0.29, test acc:0.2679\n", "epoch:11, train acc:0.31666666666666665, test acc:0.2784\n", "epoch:12, train acc:0.33666666666666667, test acc:0.2893\n", "epoch:13, train acc:0.36, test acc:0.3046\n", "epoch:14, train acc:0.36333333333333334, test acc:0.3098\n", "epoch:15, train acc:0.38333333333333336, test acc:0.322\n", "epoch:16, train acc:0.4033333333333333, test acc:0.3384\n", "epoch:17, train acc:0.41, test acc:0.3432\n", "epoch:18, train acc:0.43, test acc:0.3521\n", "epoch:19, train acc:0.44333333333333336, test acc:0.3763\n", "epoch:20, train acc:0.4766666666666667, test acc:0.3891\n", "epoch:21, train acc:0.5066666666666667, test acc:0.4154\n", "epoch:22, train acc:0.5166666666666667, test acc:0.4245\n", "epoch:23, train acc:0.5433333333333333, test acc:0.4426\n", "epoch:24, train acc:0.56, test acc:0.4539\n", "epoch:25, train acc:0.5766666666666667, test acc:0.4681\n", "epoch:26, train acc:0.5733333333333334, test acc:0.4782\n", "epoch:27, train acc:0.58, test acc:0.479\n", "epoch:28, train acc:0.59, test acc:0.4971\n", "epoch:29, train acc:0.6133333333333333, test acc:0.5048\n", "epoch:30, train acc:0.6133333333333333, test acc:0.5123\n", "epoch:31, train acc:0.6233333333333333, test acc:0.5178\n", "epoch:32, train acc:0.6133333333333333, test acc:0.5173\n", "epoch:33, train acc:0.6233333333333333, test acc:0.525\n", "epoch:34, train acc:0.63, test acc:0.5387\n", "epoch:35, train acc:0.6266666666666667, test acc:0.5391\n", "epoch:36, train acc:0.6566666666666666, test acc:0.5502\n", "epoch:37, train acc:0.64, test acc:0.5461\n", "epoch:38, train acc:0.6766666666666666, test acc:0.5576\n", "epoch:39, train acc:0.7033333333333334, test acc:0.5734\n", "epoch:40, train acc:0.7166666666666667, test acc:0.5826\n", "epoch:41, train acc:0.73, test acc:0.5899\n", "epoch:42, train acc:0.73, test acc:0.5888\n", "epoch:43, train acc:0.7333333333333333, test acc:0.5837\n", "epoch:44, train acc:0.7366666666666667, test acc:0.5961\n", "epoch:45, train acc:0.75, test acc:0.5948\n", "epoch:46, train acc:0.75, test acc:0.5988\n", "epoch:47, train acc:0.7533333333333333, test acc:0.5995\n", "epoch:48, train acc:0.75, test acc:0.5963\n", "epoch:49, train acc:0.7566666666666667, test acc:0.6032\n", "epoch:50, train acc:0.74, test acc:0.6085\n", "epoch:51, train acc:0.7466666666666667, test acc:0.6066\n", "epoch:52, train acc:0.7466666666666667, test acc:0.6071\n", "epoch:53, train acc:0.7466666666666667, test acc:0.6031\n", "epoch:54, train acc:0.7466666666666667, test acc:0.6062\n", "epoch:55, train acc:0.7633333333333333, test acc:0.6192\n", "epoch:56, train acc:0.7633333333333333, test acc:0.6096\n", "epoch:57, train acc:0.7633333333333333, test acc:0.611\n", "epoch:58, train acc:0.77, test acc:0.6273\n", "epoch:59, train acc:0.7833333333333333, test acc:0.622\n", "epoch:60, train acc:0.7866666666666666, test acc:0.6271\n", "epoch:61, train acc:0.78, test acc:0.6324\n", "epoch:62, train acc:0.7766666666666666, test acc:0.6379\n", "epoch:63, train acc:0.78, test acc:0.6342\n", "epoch:64, train acc:0.7866666666666666, test acc:0.6417\n", "epoch:65, train acc:0.7933333333333333, test acc:0.6449\n", "epoch:66, train acc:0.8, test acc:0.6452\n", "epoch:67, train acc:0.8, test acc:0.6521\n", "epoch:68, train acc:0.7966666666666666, test acc:0.6393\n", "epoch:69, train acc:0.8, test acc:0.6382\n", "epoch:70, train acc:0.78, test acc:0.6444\n", "epoch:71, train acc:0.78, test acc:0.6461\n", "epoch:72, train acc:0.78, test acc:0.6451\n", "epoch:73, train acc:0.7833333333333333, test acc:0.6578\n", "epoch:74, train acc:0.8133333333333334, test acc:0.6643\n", "epoch:75, train acc:0.8033333333333333, test acc:0.6597\n", "epoch:76, train acc:0.8133333333333334, test acc:0.6635\n", "epoch:77, train acc:0.8133333333333334, test acc:0.6614\n", "epoch:78, train acc:0.81, test acc:0.6584\n", "epoch:79, train acc:0.82, test acc:0.657\n", "epoch:80, train acc:0.81, test acc:0.6598\n", "epoch:81, train acc:0.8133333333333334, test acc:0.6648\n", "epoch:82, train acc:0.8166666666666667, test acc:0.6698\n", "epoch:83, train acc:0.8166666666666667, test acc:0.673\n", "epoch:84, train acc:0.81, test acc:0.6522\n", "epoch:85, train acc:0.8166666666666667, test acc:0.6637\n", "epoch:86, train acc:0.8133333333333334, test acc:0.6721\n", "epoch:87, train acc:0.8066666666666666, test acc:0.6643\n", "epoch:88, train acc:0.8166666666666667, test acc:0.6562\n", "epoch:89, train acc:0.8266666666666667, test acc:0.6758\n", "epoch:90, train acc:0.8066666666666666, test acc:0.6624\n", "epoch:91, train acc:0.82, test acc:0.6743\n", "epoch:92, train acc:0.8033333333333333, test acc:0.6687\n", "epoch:93, train acc:0.8233333333333334, test acc:0.6741\n", "epoch:94, train acc:0.8466666666666667, test acc:0.6755\n", "epoch:95, train acc:0.8233333333333334, test acc:0.6743\n", "epoch:96, train acc:0.82, test acc:0.6528\n", "epoch:97, train acc:0.81, test acc:0.6727\n", "epoch:98, train acc:0.8266666666666667, test acc:0.6764\n", "epoch:99, train acc:0.8333333333333334, test acc:0.6712\n", "epoch:100, train acc:0.8133333333333334, test acc:0.6769\n", "epoch:101, train acc:0.8266666666666667, test acc:0.6769\n", "epoch:102, train acc:0.8233333333333334, test acc:0.6692\n", "epoch:103, train acc:0.8266666666666667, test acc:0.6814\n", "epoch:104, train acc:0.84, test acc:0.6827\n", "epoch:105, train acc:0.8366666666666667, test acc:0.6943\n", "epoch:106, train acc:0.8333333333333334, test acc:0.6793\n", "epoch:107, train acc:0.8333333333333334, test acc:0.6873\n", "epoch:108, train acc:0.8633333333333333, test acc:0.6901\n", "epoch:109, train acc:0.8433333333333334, test acc:0.6907\n", "epoch:110, train acc:0.84, test acc:0.6754\n", "epoch:111, train acc:0.82, test acc:0.6733\n", "epoch:112, train acc:0.83, test acc:0.6839\n", "epoch:113, train acc:0.8233333333333334, test acc:0.6786\n", "epoch:114, train acc:0.8266666666666667, test acc:0.6848\n", "epoch:115, train acc:0.83, test acc:0.6854\n", "epoch:116, train acc:0.8366666666666667, test acc:0.6931\n", "epoch:117, train acc:0.83, test acc:0.6771\n", "epoch:118, train acc:0.8333333333333334, test acc:0.6859\n", "epoch:119, train acc:0.8133333333333334, test acc:0.6627\n", "epoch:120, train acc:0.83, test acc:0.6808\n", "epoch:121, train acc:0.8433333333333334, test acc:0.6868\n", "epoch:122, train acc:0.8533333333333334, test acc:0.6952\n", "epoch:123, train acc:0.85, test acc:0.6834\n", "epoch:124, train acc:0.86, test acc:0.6888\n", "epoch:125, train acc:0.8466666666666667, test acc:0.6888\n", "epoch:126, train acc:0.8333333333333334, test acc:0.6753\n", "epoch:127, train acc:0.85, test acc:0.6877\n", "epoch:128, train acc:0.85, test acc:0.6861\n", "epoch:129, train acc:0.8566666666666667, test acc:0.6903\n", "epoch:130, train acc:0.89, test acc:0.6969\n", "epoch:131, train acc:0.8633333333333333, test acc:0.6727\n", "epoch:132, train acc:0.8666666666666667, test acc:0.6894\n", "epoch:133, train acc:0.8433333333333334, test acc:0.6854\n", "epoch:134, train acc:0.8533333333333334, test acc:0.688\n", "epoch:135, train acc:0.87, test acc:0.6916\n", "epoch:136, train acc:0.8466666666666667, test acc:0.6756\n", "epoch:137, train acc:0.88, test acc:0.697\n", "epoch:138, train acc:0.86, test acc:0.6944\n", "epoch:139, train acc:0.8533333333333334, test acc:0.6936\n", "epoch:140, train acc:0.8833333333333333, test acc:0.6931\n", "epoch:141, train acc:0.8733333333333333, test acc:0.6955\n", "epoch:142, train acc:0.86, test acc:0.6574\n", "epoch:143, train acc:0.8966666666666666, test acc:0.7088\n", "epoch:144, train acc:0.8833333333333333, test acc:0.7009\n", "epoch:145, train acc:0.88, test acc:0.6973\n", "epoch:146, train acc:0.88, test acc:0.6999\n", "epoch:147, train acc:0.89, test acc:0.6995\n", "epoch:148, train acc:0.89, test acc:0.6978\n", "epoch:149, train acc:0.8966666666666666, test acc:0.6997\n", "epoch:150, train acc:0.8766666666666667, test acc:0.6923\n", "epoch:151, train acc:0.87, test acc:0.6959\n", "epoch:152, train acc:0.8733333333333333, test acc:0.6889\n", "epoch:153, train acc:0.8566666666666667, test acc:0.6771\n", "epoch:154, train acc:0.8766666666666667, test acc:0.6942\n", "epoch:155, train acc:0.8633333333333333, test acc:0.694\n", "epoch:156, train acc:0.8866666666666667, test acc:0.6994\n", "epoch:157, train acc:0.88, test acc:0.7074\n", "epoch:158, train acc:0.8766666666666667, test acc:0.6934\n", "epoch:159, train acc:0.89, test acc:0.6965\n", "epoch:160, train acc:0.88, test acc:0.6968\n", "epoch:161, train acc:0.9, test acc:0.7069\n", "epoch:162, train acc:0.8833333333333333, test acc:0.691\n", "epoch:163, train acc:0.8966666666666666, test acc:0.7051\n", "epoch:164, train acc:0.87, test acc:0.6845\n", "epoch:165, train acc:0.8833333333333333, test acc:0.7016\n", "epoch:166, train acc:0.8933333333333333, test acc:0.7031\n", "epoch:167, train acc:0.8866666666666667, test acc:0.7043\n", "epoch:168, train acc:0.87, test acc:0.6842\n", "epoch:169, train acc:0.8833333333333333, test acc:0.6896\n", "epoch:170, train acc:0.88, test acc:0.6934\n", "epoch:171, train acc:0.87, test acc:0.6929\n", "epoch:172, train acc:0.8833333333333333, test acc:0.6925\n", "epoch:173, train acc:0.8966666666666666, test acc:0.696\n", "epoch:174, train acc:0.9033333333333333, test acc:0.7032\n", "epoch:175, train acc:0.8733333333333333, test acc:0.6969\n", "epoch:176, train acc:0.8766666666666667, test acc:0.7053\n", "epoch:177, train acc:0.8666666666666667, test acc:0.694\n", "epoch:178, train acc:0.8833333333333333, test acc:0.7007\n", "epoch:179, train acc:0.8966666666666666, test acc:0.7052\n", "epoch:180, train acc:0.9, test acc:0.7059\n", "epoch:181, train acc:0.89, test acc:0.7026\n", "epoch:182, train acc:0.8866666666666667, test acc:0.6923\n", "epoch:183, train acc:0.8966666666666666, test acc:0.704\n", "epoch:184, train acc:0.89, test acc:0.703\n", "epoch:185, train acc:0.8966666666666666, test acc:0.699\n", "epoch:186, train acc:0.9, test acc:0.7086\n", "epoch:187, train acc:0.8866666666666667, test acc:0.6983\n", "epoch:188, train acc:0.8933333333333333, test acc:0.7023\n", "epoch:189, train acc:0.8933333333333333, test acc:0.7025\n", "epoch:190, train acc:0.8866666666666667, test acc:0.6975\n", "epoch:191, train acc:0.8933333333333333, test acc:0.6798\n", "epoch:192, train acc:0.89, test acc:0.6974\n", "epoch:193, train acc:0.8966666666666666, test acc:0.7055\n", "epoch:194, train acc:0.9, test acc:0.7001\n", "epoch:195, train acc:0.8966666666666666, test acc:0.7021\n", "epoch:196, train acc:0.8933333333333333, test acc:0.7025\n", "epoch:197, train acc:0.8966666666666666, test acc:0.7062\n", "epoch:198, train acc:0.88, test acc:0.7079\n", "epoch:199, train acc:0.88, test acc:0.7059\n", "epoch:200, train acc:0.8933333333333333, test acc:0.7025\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xUZdbA8d9JL6RAQgkJJfQivSigKKIURUHX3t1VdNe2visq7+6qu76uuNjWXZRFFKwoKgIqCgoWpIiBEHqoCaTQU0hIz/P+cYeQMpNMymRC5nw/n3zI3DJzchOec+9TxRiDUkopz+Xl7gCUUkq5lyYCpZTycJoIlFLKw2kiUEopD6eJQCmlPJwmAqWU8nAuSwQi8raIHBWRbQ72i4i8JiJ7RWSLiAx2VSxKKaUcc+UTwXxgQjX7JwLdbV9TgTdcGItSSikHXJYIjDE/ASerOWQy8K6xrAfCRSTKVfEopZSyz8eNnx0NHCr3OsW2Lb3ygSIyFeupgeDg4CG9evVqlACVUqq52Lhx43FjTGt7+9yZCMTONrvzXRhj5gBzAIYOHWri4uJcGZdSSjU7IpLsaJ87ew2lAB3KvY4B0twUi1JKeSx3JoKlwB223kMXAFnGmCrVQkoppVzLZVVDIrIAuASIFJEU4GnAF8AYMxtYBlwB7AVOA3e7KhallFKOuSwRGGNurmG/AR5w1ecrpZRyjo4sVkopD6eJQCmlPJwmAqWU8nCaCJRSysNpIlBKKQ+niUAppTycJgKllPJwmgiUUsrDaSJQSikPp4lAKaU8nCYCpZTycJoIlFLKw2kiUEopD+fOFcqUUuqcsTg+lZnLE0nLzKN9eCDTxvdkyqBod4fVIPSJQCmlgJO5hexMz7a7b3F8KtMXbSU1Mw8DpGbmMX3RVhbHpzZukC6iiUApD7U4PpVRM1YR++RXjJqx6pwr1I6dKuD5ZTvJKyyp93sln8jlqn//zJRZa+y+38zlieQVVdyeV1TCzOWJFbYZY/h5z3Ge/XIHWXlFTn329rQs/u/LHXy9NZ3SUrvLtrv8d6VVQ0p5oDN3uGcKtzN3uEC9qjte/nY3R7LyeeG6/nb3l5Ya/rk8kaTjubxx22BEpM6f9d76ZP77035iWgUxuGM4f1qYwFt3DSM6PBCAvMISSoyhhX/1xVx2fhE3/nc9x3IKKCk1JKRkckGXiArHpGXm2T03LTOP37+/kZbBfjwxoRcPLYjnp93HAPD19uLJib04cDyX3IJitqZm8p9V+0jLzKNtWACPj+vJpkMZvL/+ICIw9+cDdG0dzJ0jO/PLgZNkni7k3d+ezxcJaS75XZWniUApD1TdHa4zhYuj+vIlm1NJPnGa3wyJYXhsq7Lj9x/L4Zvth9maksXX2w4DsC01m34xYbWKO7egmJnLd7FixxHSMvMBmLVqD93ahLDr8Cm+3X6Yu0bFUlBcwm/eWMvRUwV8cM/59GwX4jDufcdyOJydz7y7h3H3vF/ZmJxRJRGEBfmSebrqHX5IgE/Zz/PF5jROF5Xw1KQ+bDqYwfy1B8gvKmH+2qQq5x3OyudPnyZgDPx2VCwPXdqNNfuOM+v7fTy1ZDt+3l4UlpSycueRev+unKGJQCkPVN0dbk0cPU3kFxWTfOI0ADOX72LhfSMQEbJOF3Hr3F9Iz8rH20u4b3QX3ly9n5vfXEduQUm1Da8vr0ika5sWTB5o7bvvvTh+3nuiwjGHsws4nF0AwJp9J7hrVCwvrdjNjvRswoN8uXHOOpY8MIr4g5lV4n5y0RZKSg1X9o9iTM82dG/Tgl+TTlZ4/4LiEjAgAqZSzU12fjGT+kcxpFNL/rVyD6/fMJjxfdsx9kQbvtl2mPlrk/jN4BhW7TpCRqVEYgyEBvjw1FV9AJjUvz1X9otiW2o2MS0DueK11by7LrlevytnaSJQygO1Dw8k1U5B0t5WrWJPamYeQb7eDu9QX1yxG4CxvdqwctdRHvwwnqsHtmfRphSOnSpg0R9G0i86jK+2pAOQU1B9VUdqZh6vrdoLwNHsAsb3bVclCZzhJTB5YDTf7TzCxuQM3ly9n1vO78jUi7ow7tWfeHP1fr7dcaRK3PlFpQD8z+U9ABjauRVfbkmjtNTg5WVVWy389RCZeUX8/uKuLE1IIy0zj6iwAEZ2i6SguJRnJ/clPMiPu0Z2Lqvq6hQRzF8n9SGnoJg/XNKVLtOX2Y37VH5xhdciUvaUdNsFnZi5PJHwQF8y7bQ3VPe7qi1NBEp5oGnje/Lkoi1lBSFAoK8308b3dHjO7XN/oVNEkMM70eM5hQD8fcp59PolmXfWJvPV1vSyzxvcsSVgVUtVbhMtX9WxYMNBhnRqyS/7rUL//NhWPLdsJ698t9thbMbAmF5t+Dw+lYcXxNMqyI//vaI3Lfx9mNQ/ikWbUjldTaNy19YtABjWuSULNhxk3tokvATuHNGZ+WuT6B8TxuMTevLExF4O36Nye8edIzuXfV+XxHvjsA78e9Ueu0mgpt9VbWkiUMoDTRkUzaGM07xku4v3FuEf15xXpXomv6gEX28vMk4Xsv94LsknT9M2LIDDWflV3jPIz5sgP2/ahwUwbXwv7ru4K0nHcwn09aZ725Cy4xwlktTMPJJP5DJ90Vb6x4QRHuRH54ggPrz3ApYmpDJvTRJJx3PJrnQXDVaBOrJrRNn7PDWpT1kj8Z0jOrNok+NeNm1D/cu+H9bZatd49ssdAGxJyWLfsVxevH5AvRq2p43vWaFaCmouzCNb+LPyT5dwIqeAjckZzF19wGVjGDQRKOWhWgX7AXDPhbHM/fkAbUMDWLfvBCNsBWpxSSkT/7Wai3u0ZnSPSABKSg2jukawbGs6eeWeJsCqnunTPqyswAwN8KV/THiVz3V0d+zrLSzdnAZYBTDA7y6MxdtLuGZQDNcMiqnSPgFnC9TIFv70iQol83Qht5zfsWz/gA7hDOgQTmJ6FohUeAry9hKmT+xd9jqmpVXItgsN4PP4VD6PT6VlkC+T+kfV4spWdabQru2AtOjwQKLDA+kfE87do2LrFUN1NBEo5Sb/WbWHC7u3ZmCHqoVlY9hyKIuWQb7cNLwjc38+wC1zfwFg9eNj6NAqiO92HuHA8VxO5RcTGuCDl0DHVkEknzjNg5d2L+tDHxUWwMmcAnIKSujbPrTGz7V3d+zrLRSVGN5ac4DBHcPJyiti37FcxvZqU+HcmgrUWbcOBiDA17vCebNuGUROQTG70k+VnRvRwo8nJ/SqUBiLCA+M6QbAqG6RTJ71M3eM6Fzl/epiyqDoJjsSWROBUm5wJDufF1fsZkNSBu/+drhLPuPlFYnsSD/FnNuHlDV8lpeQksmADuF0bR3MtYOj8Rbhk40prNx5hLtGxfLO2mRE4HhOAQvjUujeJoSrB7Zn5vJEoltaddtnksa/vtvDK9/tdioR2CvMH760G//31U4yTxdxzeAYukYGM39tEsPKdUEtf76jAjU2Mtju9piWQQD0ahfqdGHcLiyAn5+4FF/v5j/uVhOBUm4Ql5QBwJq9xzmeU0BkC/8aznDOlpRMDtjq5c/0uFm2LZ1J/dtXOG7J5lQSj5ziyn5RiAgv3zAQgI0HM1i56yijukWybv8J7r0olrd+PsDh7HxG94jkluEdWRyfypLNabQN9SfGlhCmju5CkJ83l/Vu61Sc9grzxCM5vP9LMlec146IFv6M7BZZ38tRb56QBEATgVJu8WvSSby9hJJSw/PLdrJ+/8l6NwQePZXPnW9vKOuv3qtdCKXG8PKK3Uzo245SA498FE96Vj4JKZmcH9uKuy+sWO98We+2zFtzgOmLthLs5839F3dl86FMfk3KYECHcFoG+/HxfSN44INN9O9wtj0g0M+be0d3qdc1eXxCT24f0YmIBkqKynmeke6UqoeEQ5n0e3o5e4/mNNh7bkzOYHjnVrQL9WdRfGq9JzMzxvD4p1s4XVjCC7/px+SB7fn3zYP407ie7D+ey2ebUvh6WzpfbzuMj5dw2/mdmH/38CrTL4zt1YaiEkNccgZPX9WXiBb+ZXf5A2wNv62C/Vgw9YIKjawNIcDX22HVjnItfSJQqgZLNqdxqqCYr7ak88hl3Wt1bmmpYfZP++jZNoRLe7VBRMgpKGZ7WhYPjOnGjvSsKqNV6zLVw5leONcOas+Nwzpy4zCr10y3Ni0Y0CGcV7/bQ9vQADpHBLHwvhF22wwAhnRqSZsQfwZ3bMn1Q2MAuGNEZzq2CnKq/l+dmzQRKFUNYwwrdx0BYNWuI3YTQWmpobCk1G7Pki+2pPHPb6zeNb3ahfD7S7pSagylxhrF+h9bPX5llfva5xeV8MRnWwgL9OWpSX0oLjV8kZDGU0u2V+h98/W2w4zukVqWRESEx8f3LJvi4S9X9naYBAB8vL1Y8ehoWvj7VKj2mdivft0nVdOmiUCpauw7lkvyidN0iggiISWLo9n5tAkNKNt/7FQBd7y9AV9vYckDoyoMOioqKeWVb3fTq10I91zUhTd+2MsjH20GwMdLGNQx3GGf+jblBjkdzc7n4Y/iWb/fmgNna2oWe4/kUFhSSkFxxb78eUWlVZ4mRnWLZFS3CDYlZ3L9kA41/szhQX5OXh3VXGgiUB7PGMNrK/fSv0MYY3pW7Le+cqf1NPD0VX347fw4Vu06yk3DrWqXrLwibvzvOvYfzwVgR3o2qRl5rNt/guuGxLBoUypJJ07z1p1DGdu7LdcOimbNvuOczC0kpmUgoQG+dvvUA2TnFbF8+2F+2n2MT+JSKDWGf900kOM5hTy/bCcdWwWVfW5l9kbuvnbTII6eKiAsyLfe10s1P5oIVLORmplHVGhAtVUf9ny2KZVXvttNn6hQxvRsw9HsfAL8vAkN8OW7nUfobdseHR7I6z/so0WAD1f2i2Lhr4fYfzyXWbcM5sEPN3HD7HXk2uazmbcmCYDJA9tzqW1QlJeXcFH31hU+u3yf+jNPBqEBPoQF+XLfexvx8/biuqEx3De6C50irIbUO0Z0IjUjj0te/MHuz2Nv/pqIFv7aG0c5pIlANQs/JB7lrnm/EuzvzemCEtqHB3DVgPZ4ieAlwrWDo4mNDObjXw/RoVUQI7tGICIcOnmaZ5Zux89b2JGeTeyTXyECPduG8NZdw4hLzuDRy3ogIrzwm/48vXQbD34Yz5FJBby3PplhnVtSVFKKCGVJAKyRsg9f2p2HxtbcuHymT/3c1ft5btlO3rhtCN3atOCLhDQm9W9Pu7CACsf7envROTKYDi0DOZRR8e6/oScjU55BTOUuC03c0KFDTVxcnLvDUG4wf80BjuUUMG18L95dl8TCuEN4e3kxrk9bZv+wl1MFVWeXFKx55MOD/BjRNaJsCuShnVoy69bBPPDBJramZIJIlfr2EV1asW7/SX547BI627o1lpYa7nk3ju8Tj2IM/PvmQcz4epfdev7o8EDWPHmp0z9fcUkpKRl5ZZ9Vk7mr9/N/X+0kxN+HnILiZregumpYIrLRGDPU3j6XPhGIyATgX4A3MNcYM6PS/jDgfaCjLZYXjTHzXBmTOnfN/fkAKRl5FBaXMvfnA/RuF4qvN1XWjS0vKiyAD++9gFvn/sJXW9L5/SVdiQ4P5LmvdjL2pR/JKSimZZBvlUVDANbtP8mADuEVCmYvL+vJYMKrP+HlJYzv246HF8Tb/ezaLhziY7vTd9b1Qzqw71guj43rodU+ql5clghExBuYBVwOpAC/ishSY8yOcoc9AOwwxlwlIq2BRBH5wBhT6Kq41LnpcFY+KRl5+Hl78ebqA3SKCOKT+0cQ7O/DlpRMJv9nDfaebdOz8ukcGcznfxjJ9vTsssbg3lEh3DXvVyYPbF8246U9Vw9oX2Vb6xB/Pvv9SIpKSvHz8arTXPMNISzIl+ev7efSz1CewZVPBMOBvcaY/QAi8hEwGSifCAwQIlafuxbASaDqZOOq2cnILeThj+LZdfgUIf4+3DisAydyC1m//wRv3TmM1iEV73Djkq2ukzOv789HGw7xxMReBNtGxfaPcdwN80xh3CY0oEK3zyGdWvHL/44lwMebuKQMu+f6eXsxeWDVRABUuHOvy1zzSjUlrpxiIho4VO51im1bef8BegNpwFbgEWNMaaVjEJGpIhInInHHjh1zVbyqgSyOT2XUjFXEPvkVo2asqjJdwtFT+dw0Zz2/HDjJmJ6taRPqz/Nf72Lu6v1sScliyeaq0yvEJWUQ6OvNFf2iWDD1gipTN08b35PASgO6aiqMg/x88PISh+f+87r+Tk0GN2VQNM9f24/o8EAEq23g+Wv7aV29Ome48onAXh++yk/v44HNwKVAV+BbEVltjMmucJIxc4A5YDUWuyBW1UAcLWwOVoGZmpnHrW+u5+ipAubfNaxshsnEw6cI9vfm5jnreeGbXTz31c4KjZ9xyScZ2CHc4WyQdV34o77nln8PLfjVucqViSAFKD+MMQbrzr+8u4EZxuq6tFdEDgC9gA0ujEu50Avf7LK7sPnM5YkM7BDOrXN/ITu/iPd+dz5DOrUsO6ZnuxAWx6dyODufohIr159JIvlFJexIy+ZB24IhjtSnMNaCXHkyV1YN/Qp0F5FYEfEDbgKWVjrmIDAWQETaAj2B/S6MSblYup21bMEq1K//7zryikpYcO8FFZLAGTOXJ5YlgTPyikr48+JtlBprqgSlVMNz2ROBMaZYRB4ElmN1H33bGLNdRO637Z8NPAvMF5GtWFVJTxhjjrsqJuVa21Kzqt3v5+3F/HuGVVjIvDxH3S1LSg1v3TmU87tE1DtGpVRVLh1HYIxZBiyrtG12ue/TgHGujEE1DmMML65IJNDXCwNVFgi/dnA0z04+r9q1Xx31/IkOD2SskytfKaVqTxemUfVmjOH5r3fxQ+IxHr28BzOu7V+hB81L1w9g5nUDalwAvC49f5RS9adzDak6mfX9Xub8ZDXnlBrDqfxi7hjRiXsu7IKXl9Sp4bUheu8opWpPE4GqYmPySfx9vDkvOszu/tyCYmb/sI/OkcFljb7d27bgluEdK8zHXxfae0epxqeJwIOdWVnr4MnTzF+bRICPNx1aBfLslztoGxrA6sfH4GOn3/7izamcKijmmav7MKRTKzdErpRqSJoIPFRJqeHa19eQkGL19Anw9aKk1FBUYujYKoiDJ08z/B8rycgtpGWwH9cMiuYCW6+d+WuS6Ns+lMEdq3YBVUqdezQReKilCakkpGRx+wWd6No6mKsGtCe/uJTVu48hwJOLtnIy15r772RuIW/9fIC3fj5Qdv7M6/rXuxpIKdU0aCLwQIXFpbzy7R76tg/lb1f3rbCi103DOzJqxiq7M3m2DvFn3l3D8PX2okfbFo0XsFLKpTQReJDSUsOKHUeY9f1eDp48zby7htld1tHRwK7jpwocNiArpc5dmgg8wEsrEtl95BR7j+aw71gunSKCePH6AVzSs7Xd4901v75Syj00ETRTi+NTKyyIHhHsS9fWITxyWQ+uOK+d3d5AZ+j8+kp5Fk0EzVDlqaDBWlj9lvM72l1xqzId2KWUZ9FE0AzNXJ5YZSro/KJSZi5PdLow14FdSpUzszvkHq26PbgNTNvT+PE0ME0EzZCjxt7aLqauVK0ZA6YUvKqfV+qcYy8JVLfdGGio7tWNkIQ0ETRD2tirauSqwmXpg5C+Be78AgLDaz7eGUlrYPOHcMVM+NcAx3Hf+QW06VV1X1Ee+Nbyb78oDzIPQeseNR9rbJ2tzxT8u1fA0ofg+nnQaaS1zdH1DoqE2xdB1ICK2w/+AgkfQkpc7ZNQHWgiaIYeG9eD/1mYUGEsgDb2qgrqU7hUl0SCWsGxXfDxbXDzAvAPgVNHrEIysBV4+9QuCR3ZAQtugoJsaNu3+rhfPx/uWAodR8Da1+DgekhPsPbdvhgWTa3+cze8CXu+hUkvw6L74OBamPoDtOpa/fWYOxaO7rKS0JC74dunIO8kfH4/THkd4j9wHPfp4/Dfi+GiP0FAKBzZbiWgg2vBPxRihsKR6j++IWgiaIZ6tw/FAOGBvmTlFWljrycxBkqKwMev4d973/eQn1l9YVyQDW36QtJq6+49vBOkbbL2+wRCu37OJ6GD62HhHeAbBJHdYc2rNce4dSGc3A+rnrXi6HYZJC6D+Per/9x1r8Py6dbr1wZDSQH4tYClD9dczZW6Efpdbz0JLX3Qivfqf1vnzr8SAmoYe9PrSlj9ovV9WAcIbAljn4Lz7we/YHjG9WN3NBE0Q2v2ngDgq0cuIlqrg5qGTe/Czi/hpg+tu2KA0lKrYBWx/vPXZON8aNEOek6wv7+4AN7/DeRlwL2rwMe//nGnJ0BoDPi3sArlguzqjy/Oh2G/hahB8MPzViE79mnryeDEXkiLr/780hJ4dzJkJMGpdKtgvHkBnD4J86+o/tzwTrDzC0jbDG37wf2rrWv7xR9hy8fVn7t8OvSaBCMfgk9/B4PvgFZdYNE94F1DUu06Fq5904p9y0cQFgNdLoGSQjh12HrPGR0dn3/j+5C6CULaQZh7btY0ETRDa/Yep3NEkCaBpiLzEHz9BBSdhh2LIXY0/PwKJCywCm3fYHjYVhA4sm4WLP9fCIqAR3eAlw+83NvxXe66WVbhm7jMOnbs01bVSsJH1l1mdXavsP5d/zrs/966qx4+1UoCY/4M3z9X/fktO0PMELjtU/v7q7vD3bPCeproPh76TIbR0862NYx8CNb+2/G5E1+wqpHys+DKl8/W2fe7DjbOqz7mq/4FA28Fb194dJt1rjGQnQIxw+CTu+1fa/GCif+0jvf2gUG3nd037J7qP7PsPcS6Xm6kiaCZKSop5Zf9J7QaqDFVV+d926ew8lmrUAnvBKtfgtUvw/Hd0HuSVfAV5sJLldpv/EPh0e1Wo2PyGsg9ZjUopidYTxcJC6qvz1/5N+vf1r0gK9VKIuffB4vvr/nn+fB669+gSOgyBvZ+BwU54B8Go/7oRCKIrfkzHFn/hvXUc9MHVqFc3rj/qz4RdB1rVcOUlkD/G85u7zgSQtrDqTTH5w656+z3ZxKICFz4qPW9owb0huwd5EhwG8d/Xw1EE0EzsyUlk9zCEkZ1i3R3KJ7BmOrrnv872vp+3HNW9c+SP1h36Ld+Al0vdXx3XJAN/x5sVYkMuAkie1h1xnPHwtePg91pActp3RsG3gwjH7YK1+XT4fBW632u/g98eCPkZ1Q9zy8YJs60CtTu46w4Xu4Dh9bDgJtrbnsQbwivphqkJgd+hIufqJoEzqiuUPTxs64zWE9DZ3h5wWVPw+f31T0uR5xNAvUpzBthnIImgmZm/f6TAIywrR3g8YyBI9sgojv4BpzdXlJUsbD55G6rCmDEH2D7Yqsg6TYWti2y6p1P7oeWnWDgbdBjHOSegB9fsBohqzP4DusuOqIrFBdaTwC9r7KSQE3EG25daFXNnDHiAVj8e+h5hVXt48gD689+P/S31p30qTS45r/Q8Xx4Muns/l3L4KObrUbOR7dXbK/wibTuruPfs6pqwHGhBlb9uKNC/AxH53v7gymBwXc6PremQnHw7fa3D7gJVvzV5XfWDpWPe+unkJUCF/7R9Z/rJE0EzcyWlExiI4NpGeyCXiNNmaPqGS8fKC22uueNfcra9u1TVsPr7YshejC8EGt199u+6GzPEQAvXygtsqoVWveApJ+tRr3uW+HTu63XMUPh0C+O4xr3nNUtEKw71hvecf5neiyx6rYBN0Pb86BNb3jWyac+3wC4ZjakbIDul1fdHzvaaqcYcKP9RutLnrTaJrqOtV7bK4xf7Q+ZydDKiWohR4V5YS6c2Oe6BtOmMgK433XujqAKxzOPqXPS9rRs+rQPdXcYrnF0p9VVD6wGwayUs/sc3aGWFlv15Fs/sZ4ONr4Da/5lDRj68Aard0reSQfnFsFFj1mNh3csseqosw5ZTwhnqjB+t6L6mAMa+HchAlH9a77rrqzLxVbDq72qDP8W8Pufz1arVBYWA5f/rfpqoXb9rH/r0z7gF2z9bKrRaSJoRrJOF5GSkUff5pgI9q6ENy+1vj66FV45D17pC7MugLX/qf7ckQ9B5kHrkXzZY1a1zNQfreqh92u4Oxv717P9yHtOtJ4wvrQ1IPa/vv4/V334O/g916Wqo1UX8Auqeyxtz7O9Tz0SgXIbrRpqBtbuO86Pice42La+QN/2zWzxmLR4q3GzdS9ryP6vb1rdCzuPsvrmr/hz9ef3utLqS774fqsefMpsCGlr9U9/d7LzcQS2hNiLYd9KiB5qFZ5Qv4bA+pz7RBJs/9yqu6/tE0JDa2dLBPV5IlBuo4mgGZi/JokVO46QkmHNL3ROPREc/AU2vw/j/2GN6LRbz+9r9SW/6wurMJ74wtkqjpEPWV0qz/TOsSewJXQdYzXUXvpXKwmAlVSue9uaDsFZfa62EkG/ck8D9al7rs+5Xt5Np7652+XWGIPyDdvqnKGJ4BxnjGFjstUN8Kut6bQN9SeyRQOMKHWl0hJrPprju2HxA1CUCyFR1dTzF1kNlmcaMivXc1eesMueC//H6sc/7HcVt/e+qnax97sestNh0K21O6+58w2Aix93dxSqjjQRnOMOHM/lRG4hbUL8OXqqgD5RTfxpYPOHVrfLjCTrdWQPq4CubqAQVN+lEGquYuk0wvqqy7nl+QXDmOlVtyt1DtNEcI6LS7KeBp6dch73vbeRfjENNPVvXTnqxukfat3VL/9faD8YRj9uTanQYTjkHIVZw6t/35rqwN1VPaNUM6CJ4BwXl3ySlkG+jOvTlk/uH0GPtiE1n+RKjqp3CrKtJND7Krhu/tmJ18AavHX9O/CxVrco5Q6aCM5BpaWGv3+5g/X7T3A8p4AhnVoiIgzr3Mq9gaVuqn7/hY/CJdMrJoEzek9yTUxKqRppImjCFsenli0gHx7ky+MTenLj0I786ZMEPo9PJaZlIMdzCrmgKUwncXgbvDmm+mMue6b6/Y0wuZZSqipNBE3U4vhUpi/aWrYIfcbpIv78+TbW7zvJkoQ0/ufyHjx0aTd2pGfTvU0DVwc5s4JUShycPHB2UNWG/1oLjxTXY11kratXyi00ETRRM5fvKksCZ5QaWJKQxrg+bXno0m6IiGsGj9W0glRRnrVISRoi76AAAB6kSURBVHaqVb/fYThsWWhN7LVxfsPHo5RyKZdOMSEiE0QkUUT2isiTDo65REQ2i8h2EfnRlfGcS9Iy8x3ue/7afoir50CvzoY5VhIIjbFmwlxws7Uy1fD7HFfjaPWOUk2Wy54IRMQbmAVcDqQAv4rIUmPMjnLHhAOvAxOMMQdFREsLmyB/b3ILSqpsjwoLIMKdA8a2LLQWV+l2OUx4HuZNtCZtu/Qv0LaPVu8odQ5yZdXQcGCvMWY/gIh8BEwGdpQ75hZgkTHmIIAxppollzyHMYZAHy/yCksoLbf+SKCvN09M6NXQH2atJZuVYk0Y5l1Dkll0rzW3/4TnrQXFp+1t2HiUUo3OlYkgGjhU7nUKcH6lY3oAviLyAxAC/MsY827lNxKRqcBUgI4d67H60Tli37FcjucWcd3gaNbtP0laZh7twwOZNr5nwyxBeeowbHgTwjtYM3ImrbbtEAiqoQvqHUuh80XWqk9KqWbBlYnAXiV25fX1fIAhwFggEFgnIuuNMbsrnGTMHGAOwNChQ2tYo+/ct3LnEQAeHdeTF+uyAH1NvX6+fsJaRB0gsJU1D31Uf9j3Pez5Fkyptai6vfO7XFz7eJRSTZpTiUBEPgPeBr42xpQ6+d4pQIdyr2OAyqtHpwDHjTG5QK6I/AQMAHbjofIKS/jo10P0iQolui5JAKrv9fPDDCsJjH7c6uXTos3Z9V1jR1truyqlPIqzz/dvYNXn7xGRGSLiTEX1r0B3EYkVET/gJmBppWOWABeJiI+IBGFVHe10MqZm6R/LdnLgeC5/ubK3az7gh+etmT4vtK2j6+/mKSmUUm7n1BOBMeY74DsRCQNuBr4VkUPAm8D7xpgiO+cUi8iDwHLAG3jbGLNdRO637Z9tjNkpIt8AW4BSYK4xZluD/GTniNJSwy8HTjKoYzifx6fy3vpk7rkwlpHdnFyPtrYu+5u1SLtfsGveXyl1znG6jUBEIoDbgNuBeOAD4ELgTuASe+cYY5YByyptm13p9UxgZm2Cbk4+3ZjC459toVWwHydzC7m0VxumTejpug+88I+ue2+l1DnJ2TaCRUAv4D3gKmNMum3XxyIS56rgmjtjDPPXJhEbGUyHVkG0CfHnH9f0w89He+QopRqPs08E/zHGrLK3wxgztAHj8SgbkzPYkZ7NP67pxy3nN2C3WG9/KCmoul1H9yql7HA2EfQWkU3GmEwAEWkJ3GyMed11oTV/765LJiTAhymD2jfcm2YkW0s7jnoELv97w72vUqrZcrYO4t4zSQDAGJMB3OuakDzD0ex8lm1N5/ohHQjya8DhHKtfAvGC4VMb7j2VUs2asyWQl4iIMcZA2TxCfq4Lq/lbsOEQxaWG20d0qv+bpW+BfausRdw3vQsjHoCwmPq/r1LKIzibCJYDC0VkNtbo4PuBb1wWVTNXVFLKhxuSGd2jNbGR9ezGWXgaFt5+djH4kChrbWCllHKSs4ngCeA+4PdYU0esAOa6Kqjm7ofEYxzJLuC5KfV4GjAGTqXD2v9YSWDCC7D/Bxh+rw4SU0rVirMDykqxRhe/4dpwPMPqPccI9PVmdI/WdX+TFX+Bdf+xvh94G1xwv/WllFK15Ow4gu7A80AfIODMdmNMFxfF1ayt2Xuc87u0qvt4gZMH4JfZ0GsS9L8Rekxo2ACVUh7F2ZJoHtbTQDEwBngXa3CZqqXDWfnsO5bLqK71mELip5ng5QNXvAh9rgYfbbdXStWds20EgcaYlbaeQ8nAMyKyGtCpKmtpzd7jAIzsFlH9gY6mkg4Ih4JsOP/3EBrlggiVUp7G2USQLyJeWLOPPgikAjpMtQ7W7DtOq2A/ercLrf5AR1NJ52dC235wyRMNH5xSyiM5WzX0RyAIeBhrIZnbsCabU7WQmpnHt9uPcGG3SLy86rH4/G2fQkBYwwWmlPJoNT4R2AaP3WCMmQbkAHe7PKpmqLTU8KeFmyk1hsfG1XN20ZB2DROUUkrhxBOBMaYEGCIi9biFVV9tTWf9/pM8fVVfOkYEuTscpZQq42wbQTywREQ+AXLPbDTGLHJJVM3Q2n3HCQnw4bohTkz9YG+9YKWUchFnE0Er4ARwabltBtBE4KS4pAyGdmrpXNvAF4843qdTSSulGpizI4u1XaAeMnIL2XM0hymDoms+ODsddn4Bo/4Il//N9cEppTyesyOL52E9AVRgjPltg0fUzCyOT+XvX+4AYN6aA0SHB1afEBIWgCmFwXc0UoRKKU/nbNXQl+W+DwCuAdIaPpzmZXF8KtMXbSWvqASA4zmFTF+0FcB+Migphvj3odMoiOjamKEqpTyYs1VDn5V/LSILgO9cElEzMnN5YlkSOCOvqISZyxMrJgJjYPHvYctCMCUwelojR6qU8mR1XRqrO9CAi+w2T2mZec5t3zjPqhIacAt0HgX9rm+E6JRSyuJsG8EpKrYRHMZao0BVo314IKl2kkH78MCzLzKSYPmfocslMHkWeNVxRlKllKojp0odY0yIMSa03FePytVFqqpp43vi612xu2igrzfTxpcbWfzTTKtxWJOAUspNnCp5ROQaEQkr9zpcRKa4LqzmYcqgaAZ1CMdLrGXdosMDef7afmfbBzKSIOEjGHKXrjGslHIbZ9sInjbGfH7mhTEmU0SeBha7JqzmIzOviIt7tGbe3cOr7lz9EogXjKpmAJlSSrmYs3UR9o6ra0Ozx8gpKGbP0RwGdAivujN1k9VVdOhvIbR94wenlFI2ziaCOBF5WUS6ikgXEXkF2OjKwJqDbalZGEPVRFBaAl8+CsGtYcz/uic4pZSycTYRPAQUAh8DC4E84AFXBdVcfLPtMN5ewoCYSolg66eQvhnG/0PXFVBKuZ2zA8pygSddHEuzkpJxmg9/OcgNQ2NoFVxpTeFN70CrLnDeb9wTnFJKleNsr6FvRSS83OuWIrLcdWGd+/713R4QeHhs94o7TuyD5DUw6DbQJR6UUk2As1VDkcaYzDMvjDEZ6JrFDp3MLeTz+FRuGd6RqLDAijvj37d6Cg24xT3BKaVUJc4mglIRKZtSQkQ6Y2c2UmVZtjWd4lLDDUM7VNyRkQxxb0H38RAa5Z7glFKqEme7gP4Z+FlEfrS9Hg1MdU1I576lm9Po1qYFvaNCzm4syoeFd1jpc8LzbotNKaUqc3aKiW+AoUAiVs+hP2H1HFKVpGXmsSHpJJMHtKfCMs8JC6yeQlNeh1ax7gtQKaUqcbax+B5gJVYC+BPwHvCME+dNEJFEEdkrIg57HYnIMBEpEZHrnAu7aSouKWXG17sAuGpApUFi+1ZCWAfodaUbIlNKKcecbSN4BBgGJBtjxgCDgGPVnSAi3sAsYCLQB7hZRPo4OO4F4JzvhfTowgSWJqQxbXxPOkcGn91RWgIHfoIuF2tPIaVUk+NsG0G+MSZfRBARf2PMLhHpWcM5w4G9xpj9ACLyETAZ2FHpuIeAz7ASzTkr8fApvkhI46FLu/HAmG7WxpndIffo2YPi37e+gtvAtD3uCVQppSpx9okgxTaOYDHwrYgsoealKqOBQ+Xfw7atjIhEYy17Obu6NxKRqSISJyJxx45V+yDiNksTUvH2Eu4c2fnsxvJJoDxH25VSyg2cHVl8je3bZ0TkeyAM+KaG0+zVgVTucvoq8IQxpkSqqTIxxswB5gAMHTq0yXVbNcawZHMao7pFEtnC393hKKVUrdR6BlFjzI81HwVYTwDlO9LHUPUpYijwkS0JRAJXiEixMeacmt5608FMUjLyePSyHu4ORSmlas2VU0n/CnQXkVggFbgJqDCc1hhT1o9SROYDX55rSQDgi4Q0/H28GNe3rbtDUUqpWnNZIjDGFIvIg1i9gbyBt40x20Xkftv+atsFzhXFJaV8uSWNsb3bEBLg6+5wlFKq1ly6uIwxZhmwrNI2uwnAGHOXK2NxlbX7TnA8p5CrK48bAPAJgOL8qtuDdZompVTToauM1dPShDRC/H24pGelwt0YCGwFMUPhxvfcE5xSSjlBE0Ed5ReV8O66JL7aks6k/lEE+HpXPODIdjiVBt3HuSU+pZRyliaCOnpxeSJzfz7Ahd0iq645ALBnhfVvt8saNzCllKolTQR1YIzh622Huax3G+be6WBA9J4V0K6/TjetlGrynB1ZrMrZdfgUqZl5XNbbQXfRLZ/AwXXQd0rjBqaUUnWgiaAOVu48AsClvez0/jm4HpY+BB1HwsiHGzkypZSqPU0EdfDdzqP0jwmjTWhAxR0/vQjzJkKL1nD9fPDWcQVKqaZPE0EtHc3OJyElk7G9KlULHdkOq56F3lfB/T9DiI4yVkqdGzQR1NKXW9IxBq7s367ijg1zrAFkk16FgDD3BKeUUnWgiaCWliSk0ScqlG5tyq1HnJcBWxZC/xsgqJX7glNKqTrQRFALySdySTiUyeSBlaaTiHsbik7D8PvcE5hSStWDJoJa+CLBmkW7wnrEOUdh9SvQYwK0O89NkSmlVN1pIqiFHxKPMSAmjPbhgWc3rnrWmlhu3HPuC0wppepBE4GTcgqK2Xwok1HdIq0NRXnw5aOw6V04/z6I7ObeAJVSqo50igknbThwguJSczYRLP+z1TYw6hG49K/uDU4ppepBnwicsDg+lYcWxAMw7ZMEvl6XAPHvw+A74PK/68AxpdQ5TZ8IarA4PpXpi7aSV1QCQFpWPklfv4rxKkR0CgmlVDOgTwQ1mLk8sSwJAISSw02ygtVewyDSzvTTSil1jtFEUIO0zLwKr//i8wEhnOaFPJ1ZVCnVPGgiqEEL/7O1Zxd5beEGnx+ZUzKJzLDeboxKKaUajrYRVGNj8klWcS+tA7IqbP+Dz1J+a9YA+90TmFJKNSB9InAgr7CERz9OoLVk2d0fUHCikSNSSinX0ETgwOLNqRw8edrdYSillMtpIrDDGMM7a5PoHRXq7lCUUsrlNBHYEZecwa7Dp7hjRCd3h6KUUi6nicCOt1YfIDTAhykd82o+WCmlznGaCCrZmpLFN9sPM3V4JIELbwLE/oHBdhauV0qpc5B2H61k5opEwoN8uZfP4OR+uGsZdB7l7rCUUspl9ImgnHlrDvDT7mM8McwH/7g5MOg2TQJKqWZPnwhsZn2/l5nLExnfpw03Hn0OfANh7NPuDksppVzO458IjDH885tdzFyeyJSB7Xm9zw68DnwPY5+CFtoOoJRq/jz+iSAuOYPXf9jHzcM78NxlbfGadTV0vgiG/s7doSmlVKPw+CeCzQczAZg2vhdeG2ZDwSmY9Ap4efylUUp5CI8v7banZREVFkArnwKIewt6X6XrDCilPIpLE4GITBCRRBHZKyJP2tl/q4hssX2tFZEBrozHnu1p2fSJCrUWoc/PstYgVkopD+KyRCAi3sAsYCLQB7hZRPpUOuwAcLExpj/wLDDHVfHYk1dYwr5jOfSNagEb3oQOF0DM0MYMQSml3M6VTwTDgb3GmP3GmELgI2By+QOMMWuNMRm2l+uBGBfGU8Wuw9mUGrjQdzdkHIChv23Mj1dKqSbBlYkgGjhU7nWKbZsjvwO+trdDRKaKSJyIxB07dqzBAtyRng1A3yOLwT8M+lzdYO+tlFLnClcmAnuT9Bi7B4qMwUoET9jbb4yZY4wZaowZ2rp16wYLcHtaNtEB+QTt/Qr6XWcNIlNKKQ/jykSQAnQo9zoGSKt8kIj0B+YCk40xjbrs17bULG5suQcpKYABNzfmRyulVJPhykTwK9BdRGJFxA+4CVha/gAR6QgsAm43xux2YSxVnC4sZntaNqP9doF/KLQf1Jgfr5RSTYbLRhYbY4pF5EFgOeANvG2M2S4i99v2zwaeAiKA10UEoNgY0yjddjYfzKSk1NDj9GboOAK8PX6QtVLKQ7m09DPGLAOWVdo2u9z39wD3uDIGR35NyqCtZBB06gBcoL2FlFKey2Nvg+OST3JNywNwGoi9yN3hKKVcrKioiJSUFPLz890diksFBAQQExODr6+v0+d4ZCIoLillU3IGD7feAyWh0K6/u0NSSrlYSkoKISEhdO7cGVtVdLNjjOHEiROkpKQQGxvr9HkeOdfQrsOnOF1YxHmnf7FmGvXydndISikXy8/PJyIiotkmAQARISIiotZPPR6ZCHYfOcVwSSQw7zCcd627w1FKNZLmnATOqMvP6JGJ4NDJPCb7rMH4BkPPie4ORyml3MojE0HqiSyu9N6A9LoS/ILdHY5SqglaHJ/KqBmriH3yK0bNWMXi+NR6vV9mZiavv/56rc+74ooryMzMrNdn18QjE0H44bWEkWNNK6GUUpUsjk9l+qKtpGbmYYDUzDymL9par2TgKBGUlJRUe96yZcsIDw+v8+c6wyN7DXXNXk+R+OEbe7G7Q1FKucHfvtjOjrRsh/vjD2ZSWFJaYVteUQmPf7qFBRsO2j2nT/tQnr6qr8P3fPLJJ9m3bx8DBw7E19eXFi1aEBUVxebNm9mxYwdTpkzh0KFD5Ofn88gjjzB16lQAOnfuTFxcHDk5OUycOJELL7yQtWvXEh0dzZIlSwgMrP8caR73RFBQXMKgos2khQ0C3wB3h6OUaoIqJ4GatjtjxowZdO3alc2bNzNz5kw2bNjAc889x44dOwB4++232bhxI3Fxcbz22mucOFF16rU9e/bwwAMPsH37dsLDw/nss8/qHE95HvdEcPjQfnp4pbK1vU4yp5Snqu7OHWDUjFWkZuZV2R4dHsjH941okBiGDx9eoa//a6+9xueffw7AoUOH2LNnDxERERXOiY2NZeDAgQAMGTKEpKSkBonF454ITu9aCYB0G+PmSJRSTdW08T0J9K04vijQ15tp43s22GcEB5/tqPLDDz/w3XffsW7dOhISEhg0aJDdsQD+/v5l33t7e1NcXNwgsXjcE0HAwR85ZkJp3XWwu0NRSjVRUwZZa2jNXJ5IWmYe7cMDmTa+Z9n2uggJCeHUqVN292VlZdGyZUuCgoLYtWsX69evr/Pn1EXzTwQzu0Pu0bKXsQACZs4AmLbHbWEppZq2KYOi61XwVxYREcGoUaM477zzCAwMpG3btmX7JkyYwOzZs+nfvz89e/bkggsuaLDPdYYYY3fRsCZr6NChJi4uzvkTngmrZl9W/QNSSp0Tdu7cSe/evd0dRqOw97OKyEZH0/x7XBuBUkqpijQRKKWUh9NEoJRSHk4TgVJKebhmnwiOGfuNxY62K6WUp2n23UenBM53OEJwjRviUUqppqbZPxE0xghBpVQzM7O71fW88tfM7nV+y7pOQw3w6quvcvr06Tp/dk2afSKYMiia56/tR3R4IIL1JPD8tf0adKCIUqqZKTcI1antTmjKiaDZVw1Bw48QVEqd475+Eg5vrdu58660v71dP5g4w+Fp5aehvvzyy2nTpg0LFy6koKCAa665hr/97W/k5uZyww03kJKSQklJCX/96185cuQIaWlpjBkzhsjISL7//vu6xV0Nj0gESinlbjNmzGDbtm1s3ryZFStW8Omnn7JhwwaMMVx99dX89NNPHDt2jPbt2/PVV18B1hxEYWFhvPzyy3z//fdERka6JDZNBEopz1PNnTtQ/dQ0d39V749fsWIFK1asYNCgQQDk5OSwZ88eLrroIh577DGeeOIJJk2axEUXXVTvz3KGJgKllGpkxhimT5/OfffdV2Xfxo0bWbZsGdOnT2fcuHE89dRTLo+n2TcWK6VUrQW3qd12J5Sfhnr8+PG8/fbb5OTkAJCamsrRo0dJS0sjKCiI2267jccee4xNmzZVOdcV9IlAKaUqc8EU9eWnoZ44cSK33HILI0ZYq521aNGC999/n7179zJt2jS8vLzw9fXljTfeAGDq1KlMnDiRqKgolzQWN/9pqJVSCp2GWqehVkop5ZAmAqWU8nCaCJRSHuNcqwqvi7r8jJoIlFIeISAggBMnTjTrZGCM4cSJEwQEBNTqPO01pJTyCDExMaSkpHDs2DF3h+JSAQEBxMTE1OocTQRKKY/g6+tLbGysu8NoklxaNSQiE0QkUUT2isiTdvaLiLxm279FRAa7Mh6llFJVuSwRiIg3MAuYCPQBbhaRPpUOmwh0t31NBd5wVTxKKaXsc+UTwXBgrzFmvzGmEPgImFzpmMnAu8ayHggXkSgXxqSUUqoSV7YRRAOHyr1OAc534phoIL38QSIyFeuJASBHRBLrGFMkcLyO57pSU40Lmm5sGlftaFy10xzj6uRohysTgdjZVrnfljPHYIyZA8ypd0AicY6GWLtTU40Lmm5sGlftaFy142lxubJqKAXoUO51DJBWh2OUUkq5kCsTwa9AdxGJFRE/4CZgaaVjlgJ32HoPXQBkGWPSK7+RUkop13FZ1ZAxplhEHgSWA97A28aY7SJyv23/bGAZcAWwFzgN3O2qeGzqXb3kIk01Lmi6sWlctaNx1Y5HxXXOTUOtlFKqYelcQ0op5eE0ESillIfzmERQ03QXjRhHBxH5XkR2ish2EXnEtv0ZEUkVkc22ryvcEFuSiGy1fX6cbVsrEflWRPbY/m3ZyDH1LHdNNotItoj80R3XS0TeFpGjIrKt3DaH10dEptv+3hJFZHwjxzVTRHbZpm75XETCbds7i0heues2u5Hjcvh7c/P1+rhcTEkistm2vTGvl6OywfV/Y8aYZv+F1Vi9D+gC+AEJQB83xRIFDLZ9HwLsxpqC4xngMTdfpyQgstK2fwJP2r5/EnjBzb/Hw1gDYxr9egGjgcHAtpquj+13mgD4A7G2vz/vRoxrHOBj+/6FcnF1Ln+cG66X3d+bu69Xpf0vAU+54Xo5Khtc/jfmKU8Ezkx30SiMMenGmE22708BO7FGUzdVk4F3bN+/A0xxYyxjgX3GmGR3fLgx5ifgZKXNjq7PZOAjY0yBMeYAVs+44Y0VlzFmhTGm2PZyPdYYnUbl4Ho54tbrdYaICHADsMAVn12dasoGl/+NeUoicDSVhVuJSGdgEPCLbdODtkf5txu7CsbGACtEZKNtWg+AtsY2tsP2bxs3xHXGTVT8D+ru6wWOr09T+pv7LfB1udexIhIvIj+KyEVuiMfe762pXK+LgCPGmD3ltjX69apUNrj8b8xTEoFTU1k0JhFpAXwG/NEYk40182pXYCDWXEsvuSGsUcaYwVizwj4gIqPdEINdYg1KvBr4xLapKVyv6jSJvzkR+TNQDHxg25QOdDTGDAL+B/hQREIbMSRHv7cmcb2Am6l4s9Ho18tO2eDwUDvb6nTNPCURNKmpLETEF+sX/YExZhGAMeaIMabEGFMKvImLHourY4xJs/17FPjcFsMRsc0Ia/v3aGPHZTMR2GSMOWKL0e3Xy8bR9XH735yI3AlMAm41tkplWzXCCdv3G7HqlXs0VkzV/N6awvXyAa4FPj6zrbGvl72ygUb4G/OURODMdBeNwlYH+Raw0xjzcrnt5affvgbYVvlcF8cVLCIhZ77HamzchnWd7rQddiewpDHjKqfCnZq7r1c5jq7PUuAmEfEXkVisNTc2NFZQIjIBeAK42hhzutz21mKtFYKIdLHFtb8R43L0e3Pr9bK5DNhljEk5s6Exr5ejsoHG+BtrjNbwpvCFNZXFbqyM/mc3xnEh1uPbFmCz7esK4D1gq237UiCqkePqgtUDIQHYfuYaARHASmCP7d9WbrhmQcAJIKzctka/XliJKB0owrob+1111wf4s+3vLRGY2Mhx7cWqPz7zNzbbduxvbL/fBGATcFUjx+Xw9+bO62XbPh+4v9KxjXm9HJUNLv8b0ykmlFLKw3lK1ZBSSikHNBEopZSH00SglFIeThOBUkp5OE0ESinl4TQRKOViInKJiHzp7jiUckQTgVJKeThNBErZiMhtIrLBNu/8f0XEW0RyROQlEdkkIitFpLXt2IEisl7Ozvff0ra9m4h8JyIJtnO62t6+hYh8KtYaAR/YRpEiIjNEZIftfV5004+uPJwmAqUAEekN3Ig18d5AoAS4FQjGmuNoMPAj8LTtlHeBJ4wx/bFGyp7Z/gEwyxgzABiJNYIVrJkk/4g1h3wXYJSItMKaZqGv7X3+z7U/pVL2aSJQyjIWGAL8aludaixWgV3K2UnI3gcuFJEwINwY86Nt+zvAaNtcTdHGmM8BjDH55uw8PxuMMSnGmmxtM9aCJ9lAPjBXRK4FyuYEUqoxaSJQyiLAO8aYgbavnsaYZ+wcV92cLPamBT6joNz3JVirhxVjzb75GdZiI9/UMmalGoQmAqUsK4HrRKQNlK0T2wnr/8h1tmNuAX42xmQBGeUWKbkd+NFYc8eniMgU23v4i0iQow+0zTsfZoxZhlVtNNAVP5hSNfFxdwBKNQXGmB0i8hesFdq8sGamfADIBfqKyEYgC6sdAazpgGfbCvr9wN227bcD/xWRv9ve4/pqPjYEWCIiAVhPE4828I+llFN09lGlqiEiOcaYFu6OQylX0qohpZTycPpEoJRSHk6fCJRSysNpIlBKKQ+niUAppTycJgKllPJwmgiUUsrD/T8TlYw7cqnETQAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from dataset.mnist import load_mnist\n", "from common.multi_layer_net import MultiLayerNet\n", "from common.optimizer import SGD\n", "\n", "(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True)\n", "\n", "# 過学習を再現するために、学習データを削減\n", "x_train = x_train[:300]\n", "t_train = t_train[:300]\n", "\n", "# weight decay（荷重減衰）の設定 =======================\n", "#weight_decay_lambda = 0 # weight decayを使用しない場合\n", "weight_decay_lambda = 0.1\n", "# ====================================================\n", "\n", "network = MultiLayerNet(input_size=784, hidden_size_list=[100, 100, 100, 100, 100, 100], output_size=10,\n", " weight_decay_lambda=weight_decay_lambda)\n", "optimizer = SGD(lr=0.01)\n", "\n", "max_epochs = 201\n", "train_size = x_train.shape[0]\n", "batch_size = 100\n", "\n", "train_loss_list = []\n", "train_acc_list = []\n", "test_acc_list = []\n", "\n", "iter_per_epoch = max(train_size / batch_size, 1)\n", "epoch_cnt = 0\n", "\n", "for i in range(1000000000):\n", " batch_mask = np.random.choice(train_size, batch_size)\n", " x_batch = x_train[batch_mask]\n", " t_batch = t_train[batch_mask]\n", "\n", " grads = network.gradient(x_batch, t_batch)\n", " optimizer.update(network.params, grads)\n", "\n", " if i % iter_per_epoch == 0:\n", " train_acc = network.accuracy(x_train, t_train)\n", " test_acc = network.accuracy(x_test, t_test)\n", " train_acc_list.append(train_acc)\n", " test_acc_list.append(test_acc)\n", "\n", " print(\"epoch:\" + str(epoch_cnt) + \", train acc:\" + str(train_acc) + \", test acc:\" + str(test_acc))\n", "\n", " epoch_cnt += 1\n", " if epoch_cnt >= max_epochs:\n", " break\n", "\n", "\n", "# 3.グラフの描画==========\n", "markers = {'train': 'o', 'test': 's'}\n", "x = np.arange(max_epochs)\n", "plt.plot(x, train_acc_list, marker='o', label='train', markevery=10)\n", "plt.plot(x, test_acc_list, marker='s', label='test', markevery=10)\n", "plt.xlabel(\"epochs\")\n", "plt.ylabel(\"accuracy\")\n", "plt.ylim(0, 1.0)\n", "plt.legend(loc='lower right')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "訓練データはほぼほぼ正解に近いが、テストデータの結果が良くない状態。" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Weight decay\n", "\n", "バカでっかい値\n", "\n", "特定のニューロンの重みが大きいとき、そのウェイトを小さくする。\n", "具体的には、ラムダプライムというペナルティを与える。\n", "\n", "L2のむる (重みの2乗)を損失関数に与えることで、重みが大きくなるのを抑える。\n", "\n", "ノルム = ベクトルの大きさのようなもの。\n", "ノルムを計算することで、重みデータがとれる範囲を制限している" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Dropout\n", "\n", "複雑なニューロンの一部を使わなくする。\n", "\n", "計算後にDropout率をかける。例えば出力時。\n", "\n", "テスト時はDropoutさせないが、ドロップアウトさせたときと同じスケールになるような計算を入れる。\n", "Dropoutするニューロンはランダムなので、どれがDropoutになるかはわからない。\n", "\n", "Dropout率は、だいたい0.3~0.5くらいにすることが多い。\n", "あんまり大きいと、使っていないのと同義になるので大きくすることはない。" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.3" } }, "nbformat": 4, "nbformat_minor": 4 }